# -*- coding: utf-8 -*-
from datetime import timedelta
from jms.dim import jms_dim__dim_lmdm_sys_network
from jms.dim import jms_dim__dim_lmdm_sys_second_code, jms_dim__dim_lmdm_sys_network_distributi, jms_dim__dim_lmdm_sys_first_code, jms_dim__dim_sys_network_detail_dt
from jms.dwd.oms.dwd_yl_oms_oms_waybill_incre_dt import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt
from jms.ods import jms_ods__sys_manage_region
from jms.ods import jms_ods__yl_tmsnew_tms_shipment, jms_ods__yl_tmsnew_tms_shipment_stop, jms_ods__yl_tmsnew_branch_shipment_stop, jms_ods__yl_oms_interceptorpiece, \
    jms_ods__tab_barscan_centersend, jms_ods__tab_barscan_unloading, jms_ods__tab_barscan_centerarrival, jms_ods__tab_barscan_loading
from jms.ods import jms_ods__yl_tmsnew_tms_vehicle_point, \
    jms_ods__yl_tmsnew_tms_vehicle_line, jms_ods__tab_reback_transfer_express, \
    jms_ods__tab_barscan_other, jms_ods__electronic_package_list
from utils.operators.cluster_for_spark_sql_operator import SparkSqlOperator

jms_dm__dm_cn_departure_transfer_timely_route_dt_t2 = SparkSqlOperator(
    task_id='jms_dm__dm_cn_departure_transfer_timely_route_dt_t2',
    email=['macheng@jtexpress.com', 'yl_bigdata@yl-scm.com'],
    name='jms_dm__dm_cn_departure_transfer_timely_route_dt_t2_{{ execution_date | cst_ds }}',
    email_on_retry=True,
    retries=0,
    sql='jms/dm/dm_cn_departure_transfer_timely_route_dt_t2/execute.sql',
    yarn_queue='pro',
    driver_memory='32G',
    driver_cores=8,
    executor_memory='24G',
    executor_cores=6,
    pool_slots=12,
    pool='unlimited_pool',
    num_executors=95,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.executor.memoryOverhead': '5G',
          'spark.dynamicAllocation.enabled': 'true',
          'spark.shuffle.service.enabled': 'true',
          'spark.dynamicAllocation.maxExecutors': 150,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.sql.shuffle.partitions': 2700
          },
    hiveconf={'hive.exec.dynamic.partition': 'true',
              'hive.exec.dynamic.partition.mode': 'nonstrict',
              'hive.exec.max.dynamic.partitions.pernode': 5000,
              'hive.exec.max.dynamic.partitions': 30000
              },
    execution_timeout=timedelta(hours=1),
    priority_weight=20,
)

jms_dm__dm_cn_departure_transfer_timely_route_dt_t2 << [
    jms_ods__yl_tmsnew_tms_shipment_stop,
    jms_ods__yl_tmsnew_tms_shipment,
    jms_dim__dim_lmdm_sys_network,
    jms_ods__yl_tmsnew_branch_shipment_stop,
    jms_dim__dim_lmdm_sys_second_code,
    jms_dim__dim_lmdm_sys_first_code,
    jms_dim__dim_lmdm_sys_network_distributi,
    jms_ods__tab_reback_transfer_express,
    jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
    jms_ods__tab_barscan_centersend,
    jms_ods__tab_barscan_unloading,
    jms_ods__tab_barscan_loading,
    jms_ods__tab_barscan_centerarrival,
    jms_ods__tab_barscan_other,
    jms_ods__electronic_package_list,
    jms_ods__yl_tmsnew_tms_vehicle_point,
    jms_ods__yl_tmsnew_tms_vehicle_line,
    jms_ods__yl_oms_interceptorpiece,
    jms_dim__dim_sys_network_detail_dt,
    jms_ods__sys_manage_region,
]
